
#@+leo-ver=5-thin
#@+node:slzatz.20120130065448.1611: * @file synchronize.py
#@@first
#@@nowrap
#@@tabwidth -4
#@@language python
#@+others
#@+node:slzatz.20120129221034.1649: ** imports
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future_builtins import * 

import lmdialogs

import os
import time
import sys
import datetime
import platform
import ConfigParser as configparser
import json
import urllib
import re

#import urllib.request, urllib.parse, urllib.error will be needed for python 3.x
import base64
from optparse import OptionParser
from functools import partial

import toodledo
import toodledo2

# all the db stuff and sqlalchemy
from lmdb import *

import lmglobals as g


#@+node:slzatz.20120201140839.1634: ** global objects from config
cg = g.config

print_ = g.logger.write

print = print_

print_("Hello from the synchronize module")

pb = g.pb

#@+node:slzatz.20120129221034.1631: ** synchronize
def synchronize(parent=None, showlogdialog=True, OkCancel=False): #checked=False
    
    nn = 0
    
    changes = [] #server changed context and folder
    tasklist= [] #server changed tasks
    deletelist = [] #server deleted tasks
    
    print_("***************** BEGIN SYNC *******************************************")

    key = get_key() #config_stuff
    if key:
        client = toodledo.ApiClient(key)
    else:
        return "No sync because no key", [], [], []

    log = ''

    server_sync = session.query(Sync).get('server')
    last_server_sync = server_sync.timestamp

    # used below to get interval since last sync and to set t.added in client_tasks->server; called again to get new synch time
    current_server_time = client.getServerInfo().date

    client_sync = session.query(Sync).get('client')
    last_client_sync = client_sync.timestamp


    log+= "Local Time is {0}\n\n".format(datetime.datetime.now())
    delta = datetime.datetime.now()-last_client_sync
    log+= "The last time client was synced was {0}, which was {1} days and {2} minutes ago.\n".format(last_client_sync.isoformat(b' ')[:19], delta.days, delta.seconds/60)
    delta = current_server_time-last_server_sync
    log+= "The last time server was synced was {0}, which was {1} days and {2} minutes ago.\n".format(last_server_sync.isoformat(b' ')[:19], delta.days, delta.seconds/60)

    account_info = client.getAccountInfo()

    log+= "The last task edited on server was at {0}.\n".format(account_info.lastaddedit)
    log+= "The last task deleted on server was at {0}.\n\n".format(account_info.lastdelete)

    # changes from server #################################################

    if account_info.lastcontextedit and account_info.lastcontextedit > last_server_sync:
        server_contexts = client.getContexts()
        nn+=len(server_contexts)
        log+="There were change(s) to server Contexts since the last sync and all {0} Contexts were downloaded.\n".format(len(server_contexts))
        changes.append('contexts')
    else:
        server_contexts = []
        log+="There were no Context changes (edits, deletions, additions) on the server since the last sync.\n"

    if account_info.lastfolderedit and account_info.lastfolderedit > last_server_sync:
        server_folders = client.getFolders()
        nn+=len(server_folders)
        log+="There were change(s) to server Folders since the last sync and all {0} Folders were downloaded.\n".format(len(server_folders))
        changes.append('folders')
    else:
        server_folders = []
        log+="There were no Folder changes (edits, deletions, additions) on the server since the last sync.\n"

    if account_info.lastaddedit and account_info.lastaddedit > last_server_sync:
        server_updated_tasks = []
        n= 0
        while 1:
            tasks = client.getTasks(start=str(n), end=str(n+1000), modafter=urllib.quote(last_server_sync.isoformat(b' '),b':')) #leaves colons unchanged but will %20 space
            if not tasks:
                break
            server_updated_tasks.extend(tasks)
            n+=1000

        # server_updated_tasks = client.getTasks(modafter=urllib.quote(last_server_sync.isoformat(b' '),b':')) #leaves colons unchanged but will %20 space
        nn+=len(server_updated_tasks)
        log+="Created and/or modified Server Tasks since the last sync: {0}.\n".format(len(server_updated_tasks))
    else:
        server_updated_tasks = []
        log+="There were no Tasks created and/or modified on the server since the last sync.\n"

    if account_info.lastdelete and account_info.lastdelete > last_server_sync:
        server_deleted_tasks = client.getDeleted(after=urllib.quote(last_server_sync.isoformat(b' '),b':'))
        nn+=len(server_deleted_tasks)
        log+="Server Tasks deleted since the last sync: {0}.\n\n".format(len(server_deleted_tasks))
    else:
        server_deleted_tasks = []
        log+="There were no Tasks that were deleted on the server since the last sync.\n\n"

    # changes from client #################################################
    
    client_new_contexts = session.query(Context).filter(Context.created > last_client_sync).all()
    if client_new_contexts:
        nn+=len(client_new_contexts)
        log+= "New client Contexts added since the last sync: {0}.\n".format(len(client_new_contexts))
    else:
        log+="There were no new client Contexts added since the last sync.\n"   
            
    alternate_client_new_contexts = session.query(Temp_tid).filter_by(type_='context').all()
    
    if alternate_client_new_contexts:
        log+= "Alternate method: New client Contexts added since the last sync: {0}.\n".format(len(alternate_client_new_contexts))
    else:
        log+="Alternate method: There were no new client Contexts added since the last sync.\n\n"        
    
    ###############################################################
    
    client_new_folders = session.query(Folder).filter(Folder.created > last_client_sync).all() 
    if client_new_folders:
        nn+=len(client_new_folders) 
        log+= "New client Folders added since the last sync: {0}.\n".format(len(client_new_folders))
    else:
        log+="There were no new client Folders added since the last sync.\n"    

    alternate_client_new_folders = session.query(Temp_tid).filter_by(type_='folder').all()
    
    if alternate_client_new_folders:
        log+= "Alternate method: New client Folders added since the last sync: {0}.\n".format(len(alternate_client_new_folders))
    else:
        log+="Alternate method: There were no new client Folders added since the last sync.\n\n"        
    
    ###############################################################

    # get client tasks
    client_new_tasks = session.query(Task).filter(and_(Task.tid==None, Task.deleted==False)).all()
    if client_new_tasks:
        nn+=len(client_new_tasks)
        log+="New client Tasks added since the last sync: {0}.\n".format(len(client_new_tasks))
    else:
        log+="There were no new client Tasks added since the last sync.\n" 
        
    alternate_client_new_tasks = session.query(Task).filter(and_(Task.created > last_client_sync, Task.deleted==False)).all() 
    if alternate_client_new_tasks:
        log+= "Alternate method: New client Tasks added since the last sync: {0}.\n".format(len(alternate_client_new_tasks))
    else:
        log+="Alternate method: There were no new client Tasks added since the last sync.\n\n"        
        
    
    ###########################################################################################################

    #client_updated_tasks = session.query(Task).filter(and_(Task.modified > last_client_sync, Task.deleted==False)).all()
    #client_edited_tasks = set(client_updated_tasks) - set(client_new_tasks)
    #The below should be the same as the two lines above
    client_edited_tasks = session.query(Task).filter(and_(Task.modified > last_client_sync, Task.deleted==False, Task.tid!=None)).all()
    
    if client_edited_tasks:
        nn+=len(client_edited_tasks)
        log+="Edited client Tasks since the last sync: {0}.\n".format(len(client_edited_tasks))
    else:
        log+="There were no edited client Tasks since the last sync.\n" 

    client_deleted_tasks = session.query(Task).filter(Task.deleted==True).all()
    #client_deleted_tasks = session.query(Task).filter(and_(Task.modified > last_client_sync, Task.deleted==True, Task.tid != None)).all()
    if client_deleted_tasks:
        nn+=len(client_deleted_tasks)
        log+="Deleted client Tasks since the last sync: {0}.\n".format(len(client_deleted_tasks))
    else:
        log+="There were no client Tasks deleted since the last sync.\n" 

    
    log+="\nThe total number of changes is {0}.\n".format(nn)
     
    if showlogdialog and OkCancel:
    
        dlg = lmdialogs.SynchResults("Synchronization Results", log, parent=parent, OkCancel=True)
    
        if not dlg.exec_():
            return "Canceled sync", [], [], []
    
    pb.setRange(0, nn)
    nnn=0
    pb.setValue(nnn)
    pb.show()
    
    # Note: might want to check whether a new folder on client had been simultaneously created on the server - decided not to    

    #@+<< server_contexts -> client_contexts >>
    #@+node:slzatz.20120129221034.1632: *3* << server_contexts -> client_contexts >>
    for c in server_contexts:
        try:
            context = session.query(Context).filter_by(tid=c.id).one()
            
        except sqla_exc.NoResultFound:
            
            context = Context()
            session.add(context)
            context.tid = c.id 
            
            log += "{title} is a new context received from the server\n".format(title=c.title)

        context.title = c.title

        session.commit()
        
        nnn+=1
        pb.setValue(nnn)


    #@-<< server_contexts -> client_contexts >>
    #@+<< client_contexts -> server_contexts >>
    #@+node:slzatz.20120129221034.1633: *3* << client_contexts -> server_contexts >>
    for c in client_new_contexts: # this is where we could check for simultaneous creation of folders by checking for title in server_folders
        temp_tid = c.tid

        try:
            server_tid = client.addContext(title=urllib.quote(c.title))
            log+= "Context {title} was added to server and received tid: {tid}\n".format(title=c.title, tid=server_tid)
        except toodledo.ToodledoError as value:
            print_(value)
            print_("There was a problem adding context",c.title," to the server")

        else:
            c.tid = server_tid
            #need to update all tasks that used the temp_tid
            log+= "\nClient tasks that were updated with context id (tid) obtained from server:\n"
            tasks_with_temp_tid = session.query(Task).filter_by(context_tid=temp_tid)
            #tasks_with_temp_tid = c.tasks #may be a better way to go but would have to move higher
            for t in tasks_with_temp_tid:
                t.context_tid = server_tid
                log+= "{title} is in context {context}".format(title=t.title, context=t.context.title) 

            session.commit() # new/updated context and resulting task tid changes commit - not sure this is necessary here; need to think about commits

        nnn+=1
        pb.setValue(nnn)

    #note these are from class Temp_tid and not class Context
    for c in alternate_client_new_contexts: 
        log+="alternative method: new context: {0}".format(c.title)
        session.delete(c)
        session.commit()
    #@-<< client_contexts -> server_contexts >>
    #@+<< server_deleted_contexts >>
    #@+node:slzatz.20120129221034.1634: *3* << server_deleted_contexts >>
    #server_contexts = client.getContexts()
    if server_contexts:
        server_context_tids = set([sc.id for sc in server_contexts])
        client_context_tids = set([cc.tid for cc in session.query(Context).filter(Context.tid!=0)])

        client_not_server = client_context_tids - server_context_tids

        for tid in client_not_server:
            cc = session.query(Context).filter_by(tid=tid).one()
            tasks = session.query(Task).filter_by(context_tid=tid).all()
            title = cc.title
            session.delete(cc)
            session.commit()
            #Note that the delete sets context_tid=None for all tasks in the context
            #I wonder how you set to zero - this is done "manually" below
            log+= "Deleted client context tid: {tid}  - {title}".format(title=title, tid=tid) # new server folders

            #These tasks are marked as changed by server so don't need to do this
            #They temporarily pick of context_tid of None after the context is deleted on the client
            #When tasks are updated later in sync they pick up correct folder_tid=0

            log+= "\nClient tasks that should be updated with context tid = 0 because the Context was deleted from the server:\n"
            
            for t in tasks:
                log+="{title} should to be changed from context_tid: {tid} to 'No Context'\n".format(tid=t.context_tid, title=t.title)
            
            nnn+=1
            pb.setValue(nnn)
            
            
           

    #@-<< server_deleted_contexts >>
    #@+<< client_deleted_contexts >>
    #@+node:slzatz.20120129221034.1635: *3* << client_deleted_contexts >>
    #no code for client deleted contexts yet

    #@-<< client_deleted_contexts >>
    #@+<< server_folders -> client_folders >>
    #@+node:slzatz.20120129221034.1636: *3* << server_folders -> client_folders >>
    for f in server_folders:
        try:
            folder = session.query(Folder).filter_by(tid=f.id).one()
            
        except sqla_exc.NoResultFound:
            
            folder = Folder()
            session.add(folder)
            folder.tid = f.id
            log+= "New folder created on client with tid: {tid}; {title}\n".format(tid=f.id, title=f.title) # new server folders

        folder.title = f.title
        folder.archived = f.archived
        folder.private = f.private
        folder.order= f.order 

        session.commit()
        
        nnn+=1
        pb.setValue(nnn)

    #@-<< server_folders -> client_folders >>
    #@+<< client_folders -> server_folders >>
    #@+node:slzatz.20120129221034.1637: *3* << client_folders -> server_folders >>
    for f in client_new_folders: # this is where we could check for simultaneous creation of folders by checking for title in server_folders
        temp_tid = f.tid

        try:
            server_tid = client.addFolder(title=urllib.quote(f.title))      
                 
        except toodledo.ToodledoError as value:
            
            print_(value)
            print_("There was a problem adding folder",f.title," to the server")

        else:
            #tasks_with_temp_tid = f.tasks #see below could use this before you change the folders tid
            log+= "Folder {title} was added to server and received tid: {tid}\n".format(title=f.title, tid=server_tid)

            f.tid = server_tid
            session.commit()

            #need to update all tasks that used the temp_tid
            log+= "Client tasks that were updated with folder id (tid) obtained from server:\n"

            tasks_with_temp_tid = session.query(Task).filter_by(folder_tid=temp_tid) #tasks_with_temp_tid = f.tasks may be a better way to go but would have to move higher

            # I don't think the below ever gets transmitted to the server (and it shouldn't) since it's between old sync and new
            for t in tasks_with_temp_tid:
                t.folder_tid = server_tid
                session.commit() 

                log+= "Task {title} in folder {folder} \n".format(title=t.title[:30], folder=t.folder.title)

        nnn+=1
        pb.setValue(nnn)

    #note these are from class Temp_tid and not class Folder
    for f in alternate_client_new_folders:
        log+="alternative method: new folder: {0}".format(f.title)
        session.delete(f)
        session.commit()

    #@-<< client_folders -> server_folders >>
    #@+<< server_deleted_folders >>
    #@+node:slzatz.20120129221034.1638: *3* << server_deleted_folders >>
    #server_folders = client.getFolders()
    if server_folders:
        server_folder_tids = set([sf.id for sf in server_folders])
        client_folder_tids = set([cf.tid for cf in session.query(Folder).filter(Folder.tid!=0)])

        client_not_server = client_folder_tids - server_folder_tids

        for tid in client_not_server:
            cf = session.query(Folder).filter_by(tid=tid).one()
            tasks = session.query(Task).filter_by(folder_tid=tid).all() #seems like it needs to be here
            title = cf.title
            session.delete(cf)
            session.commit()
            log+= "Deleted client folder tid: {tid}  - {title}".format(title=title, tid=tid) # new server folders

            #These tasks are marked as changed by server so don't need to do this
            #They temporarily pick of folder_tid of None after the folder is deleted on the client
            #When tasks are updated later in sync they pick up correct folder_tid=0

            log+= "\nClient tasks that should be updated with folder tid = 0 because Folder was deleted from the server:\n"
            
            for t in tasks:
                log+="{title} should to be changed from folder_tid: {tid} to 'No Folder'\n".format(tid=t.folder_tid, title=t.title)
            
            
            nnn+=1
            pb.setValue(nnn)
            
    #@-<< server_deleted_folders >>
    #@+<< client_deleted_folders >>
    #@+node:slzatz.20120129221034.1639: *3* << client_deleted_folders >>
    #no code for client deleted folders yet

    #@-<< client_deleted_folders >>
    #@+<< server_tasks -> client_tasks >>
    #@+node:slzatz.20120129221034.1640: *3* << server_tasks -> client_tasks >>
    # Update client with server changes - both new and edited tasks
    if server_updated_tasks:
        log+= "\nUpdated/created Tasks from server that were updated/created on client:\n"

    for t in server_updated_tasks:
        task = session.query(Task).filter_by(tid=t.id).first()
        if not task:
            action = "created"
            task = Task()
            session.add(task)
            task.tid = t.id
        else:
            action = "updated"
            if task in client_edited_tasks:
                client_edited_tasks.remove(task) # server changes win

        task.context_tid = t.context_id
        task.duedate = t.duedate
        task.folder_tid = t.folder
        task.title = t.title
        task.added = t.added
        task.star = t.star
        task.priority = t.priority
        task.tag = t.tag
        task.completed = t.completed
        task.note = t.note

        try:
            task.parent_tid = t.parent #only in pro accounts
        except:
            pass

        session.commit() #new/updated client task commit

        log+="{action}: tid: {tid}; star: {star}; priority: {priority}; completed: {completed}; title: {title}\n".format(action=action, tid=t.id, star=t.star, priority=t.priority, completed=t.completed, title=t.title[:30])

        if task.tag:
            for tk in task.taskkeywords:
                session.delete(tk)
            session.commit()

            for kwn in task.tag.split(','):
                keyword = session.query(Keyword).filter_by(name=kwn).first()
                if keyword is None:
                    keyword = Keyword(kwn)
                    session.add(keyword)
                    session.commit()
                tk = TaskKeyword(task,keyword)
                session.add(tk)

            session.commit()
            
        tasklist.append(task)
        nnn+=1
        pb.setValue(nnn)

    #@-<< server_tasks -> client_tasks >>
    #@+<< client_tasks -> server_tasks >>
    #@+node:slzatz.20120129221034.1641: *3* << client_tasks -> server_tasks >>
    # update server with tasks edited on client
    # probably should have urllib2.quote take place in toodledo and not individually below
    # note you can't do quote twice because, for example, %20 will become %2520, which is not good
    if client_edited_tasks:
        log+= "\nUpdated Tasks from the client that were updated on the server:\n"

    for t in client_edited_tasks:

        kwargs = dict(
                      #title=urllib.quote(t.title), 
                      title=urllib.quote(t.title.encode('utf-8')), # not sure why
                      tag=urllib.quote(t.tag) if t.tag else '', 
                      folder=t.folder_tid, 
                      context=t.context_tid, 
                      parent=t.parent_tid, #? whether ignored in a non-pro acct
                      priority=t.priority, 
                      star=int(t.star),
                      duedate=t.duedate.isoformat() if t.duedate else '',

                      note = urllib.quote(t.note.encode('utf-8'), b'/<>') if t.note else '' #without encode chokes on nbsp "\xa0"  3.1 is fine
                      )

        if t.completed:
            kwargs[b'completed'] = 1
            kwargs[b'completedon'] = t.completed.isoformat()
        else:
            kwargs[b'completed'] = 0          

        try:
            result = client.editTask(t.tid, **kwargs)
            # could check if result = 1
            
            # do title = t.title because alternatives caused unicode errors
            title = t.title 
            title = title.encode('ascii', 'replace')
            kwargs['title'] = title[:40]
            
            log+="Task tid: {tid}; star: {star}; priority: {priority}; completed: {completed}; title: {title}\n".format(tid=t.tid,**kwargs)
        except toodledo.ToodledoError as value:
            print(value)
            print("Could not update",t.tid,t.title,"on server\n")

    # update server with tasks that were created on client
    #print "client_new_tasks:",client_new_tasks
    if client_new_tasks:
        log+="\nNew client tasks that were added to the server:\n"

    for t in client_new_tasks:

        kwargs = dict(
                      #title=urllib.quote(t.title), 
                      title=urllib.quote(t.title.encode('utf-8')), #######
                      tag=urllib.quote(t.tag) if t.tag else '', 
                      folder=t.folder_tid, 
                      context=t.context_tid, 
                      parent=t.parent_tid, 
                      priority=t.priority, 
                      star=int(t.star),
                      duedate=t.duedate.isoformat() if t.duedate else '',
                      note = urllib.quote(t.note.encode('utf-8'), b'/<>') if t.note else '' # without encode chokes on nbsp "\xa0"  3.1 is fine with it
                      )

        if t.completed:
            kwargs[b'completed'] = 1
            kwargs[b'completedon'] = t.completed.isoformat()
        else:
            kwargs[b'completed'] = 0          

        try:
            result = client.addTask(**kwargs)

        except toodledo.ToodledoError as value:
            print(value)
            print("There was a problem adding this task to the server:", t.title,'\n')
        else:
            #result is the official server tid that needs to be associated with the client created task
            t.tid = int(result)
            t.added = current_server_time.date() #other than within seconds of midnight, this should be correct
            session.commit()
            
            title = t.title 
            title = title.encode('ascii', 'replace')
            kwargs['title'] = title[:40]
            
            # the following line runs into unicode type problems
            log+="Task tid: {tid}; star: {star}; priority: {priority}; completed: {completed}; title: {title}\n".format(tid=t.tid,**kwargs)
                 
        nnn+=1
        pb.setValue(nnn)
    #@-<< client_tasks -> server_tasks >>
    #@+<< server_task_deletes <-> client_task_deletes >>
    #@+node:slzatz.20120129221034.1642: *3* << server_task_deletes <-> client_task_deletes >>
    ### Delete from client tasks deleted on server
    for t in server_deleted_tasks:
        task = session.query(Task).filter_by(tid=t.id).first()
        if task:
                    
            log+="Task deleted on Server deleted task on Client - id: {id_}; tid: {tid}; title: {title}\n".format(id_=task.id,tid=task.tid,title=task.title.encode('ascii', 'replace')[:30])
            
            deletelist.append(task.id)
            
            for tk in task.taskkeywords:
                session.delete(tk)
            session.commit()
        
            session.delete(task)
            session.commit() 
            
        else:
            
            log+="Task deleted on Server unsuccessful trying to delete on Client - could not find Client Task with tid = {0}\n".format(t.id)   

        nnn+=1
        pb.setValue(nnn)
        

    #### Delete from server tasks deleted on client
    for t in client_deleted_tasks:
        # need 'if' below because a task could be new and then deleted and therefore have not tid; 
        # it will be dealt with later in the for loop
        if t.tid:
            
            try:
                result = client.deleteTask(t.tid)
                print_("delete = {0}".format(result))
                # could check if result = 1
                log+= "Successfully deleted this task on server - tid: {tid} - {title}\n".format(tid=t.tid, title=t.title.encode('ascii', 'replace')[:30])
            except toodledo.ToodledoError as value:
                print_(value)
                print_("Problem deleting on server: ", t.tid, ' - ', t.title.encode('ascii', 'replace')[:30])
        
        deletelist.append(t.id) 
        
        for tk in t.taskkeywords:
            session.delete(tk)
        session.commit()
        
        session.delete(t)
        session.commit()     
        
        nnn+=1
        pb.setValue(nnn)
        


    #@-<< server_task_deletes <-> client_task_deletes >>

    server_info = client.getServerInfo()
    server_sync.timestamp = server_info.date + datetime.timedelta(seconds=5) # giving a little buffer if the db takes time to update on client or server
    client_sync.timestamp = datetime.datetime.now() + datetime.timedelta(seconds=5)

    session.commit()  

    log+= "\nNew Sync times:\n"
    log+= "server: {0}\n".format(server_sync.timestamp.isoformat(b' '))
    log+= "client: {0}\n".format(client_sync.timestamp.isoformat(b' '))
    log+= "Time is {0}\n\n".format(datetime.datetime.now())

    print_("***************** END SYNC *******************************************")
    
    pb.hide()
    
    if showlogdialog:
        dlg = lmdialogs.SynchResults("Synchronization Results", log, parent=parent)
        dlg.exec_()
        
    return log,changes,tasklist,deletelist, #config_stuff

#@+node:slzatz.20120129221034.1643: ** downloadtasksfromserver
def downloadtasksfromserver():
    '''
    sends all tasks on server down to client
    '''
    
    # I should make it possible to select only certain contexts
    print_("Starting the process of downloading tasks from server")
    
    key = get_key() 
    if key:
        client = toodledo.ApiClient(key)
    else:
        return

    account_info = client.getAccountInfo()

    # note that Toodledo API only lets you bring back 1000 tasks at a time
    server_tasks = []
    n= 0
    while 1:
        tasks = client.getTasks(start=str(n), end=str(n+1000))
        if not tasks:
            break
        server_tasks.extend(tasks)
        n+=1000

    server_contexts = client.getContexts()
    server_folders = client.getFolders()
    
    len_contexts = len(server_contexts)
    len_folders = len(server_folders)
    len_tasks = len(server_tasks)

    print_("last task edited on server:", account_info.lastaddedit)
    print_(len_contexts," server contexts were downloaded")
    print_(len_folders," server folders were downloaded")
    print_(len_tasks, "server tasks were downloaded")
    
    pb.setRange(0, len_contexts+len_folders+len_tasks)
    pb.setValue(0)
    pb.show()

    #server tasks --> client tasks
    for n,t in enumerate(server_tasks):
        
        task = Task()
        session.add(task)
        task.tid = t.id                           
        task.context_tid = t.context_id  
        task.duedate = t.duedate          
        task.folder_tid = t.folder          
        task.title = t.title
        task.added = t.added                
        task.star = t.star
        task.priority = t.priority
        task.tag = t.tag
        task.completed = t.completed    
        task.note = t.note
        #task.note = ("".join([x for x in t.note if ord(x) < 128])) if t.note else None
        try:
            task.parent_tid = t.parent #only in pro accounts
        except:
            pass

        session.commit()

        if task.tag:

            for kwn in task.tag.split(','):
                keyword = session.query(Keyword).filter_by(name=kwn).first()
                if keyword is None:
                    keyword = Keyword(kwn)
                    session.add(keyword)
                    session.commit()
                tk = TaskKeyword(task,keyword)
                session.add(tk)

            session.commit()
    
        pb.setValue(n)
    
    #server contexts --> client contexts
    for i,c in enumerate(server_contexts, n):
        context = Context()
        session.add(context)
        context.tid = c.id
        context.title = c.title
        #context.default = c.def

        session.commit()
        
        pb.setValue(i)

    #server folders --> client folders    
    for j,f in enumerate(server_folders, i):
        folder = Folder()
        session.add(folder)
        folder.tid = f.id
        folder.title = f.title
        folder.archived = f.archived
        folder.private = f.private
        folder.order= f.order

        session.commit()
        
        pb.setValue(j)

    #Update synch timestamps
    server_info = client.getServerInfo()

    server_sync = session.query(Sync).get('server') ###########
    #sync.timestamp = server_info.unixtime #? this doesn't work because I have decided that unixtime doesn't take into account time zones
    server_sync.timestamp = server_info.date + datetime.timedelta(seconds=5) 

    client_sync = session.query(Sync).get('client')
    client_sync.timestamp = datetime.datetime.now() + datetime.timedelta(seconds=5)

    session.commit() 
    
    pb.hide() 

    print_("New Sync times")
    print_('server:', server_sync.timestamp.isoformat(b' '))
    print_('client:', client_sync.timestamp.isoformat(b' '))
    
    return True

#@+node:slzatz.20120129221034.1644: ** downloadtasksfromserver (special case for transferring biopharma)
def downloadtasksfromserver___(self):
    '''
    sends all tasks on server down to client
    '''
    
    # I should make it possible to select only certain contexts
    print("Starting the process of downloading tasks from server")

    #Authenticate
    self.toodledo_authenticate()
    client = self.toodledo_client

    account_info = client.getAccountInfo()

    #Don't really need to worry about client tasks since they have tid = 0 and will sync on next sync
    #That may be true of client_folders and client_contexts -- need to check

    #client_tasks = session.query(Task).all() #you want it to execute
    #client_folders = session.query(Folder).all()
    #client_contexts = session.query(Context).all()

    # note that Toodledo API only lets you bring back 1000 tasks at a time
    server_tasks = []
    n= 0
    while 1:
        tasks = client.getTasks(start=str(n), end=str(n+1000))
        if not tasks:
            break
        server_tasks.extend(tasks)
        n+=1000

    server_contexts = client.getContexts()
    server_folders = client.getFolders()

    print("last task edited on server:", account_info.lastaddedit)
    print(len(server_contexts)," server contexts were downloaded")
    print(len(server_folders)," server folders were downloaded")
    print(len(server_tasks), "server tasks were downloaded")

    #server tasks --> client tasks
    for t in server_tasks:
        
        if t.context_id != 1633429:     ####################################
            continue                            ###################################
        
        task = Task()
        session.add(task)
        #task.tid = t.id                           ############################################
        #task.context_tid = t.context_id  ############################################
        #task.duedate = t.duedate          ###########################################
        #task.folder_tid = t.folder            ###########################################
        task.title = t.title
        task.added = t.added                
        task.star = t.star
        task.priority = t.priority
        task.tag = t.tag
        #task.completed = t.completed    ##########################################
        task.note = t.note
        #task.note = ("".join([x for x in t.note if ord(x) < 128])) if t.note else None
        try:
            task.parent_tid = t.parent #only in pro accounts
        except:
            pass

        session.commit()

        if task.tag: ######################################################

            for kwn in task.tag.split(','):
                keyword = session.query(Keyword).filter_by(name=kwn).first()
                if keyword is None:
                    keyword = Keyword(kwn)
                    session.add(keyword)
                    session.commit()
                tk = TaskKeyword(task,keyword)
                session.add(tk)

            session.commit()

    return ###############################################################
    
    #server contexts --> client contexts
    for c in server_contexts:
        context = Context()
        session.add(context)
        context.tid = c.id
        context.title = c.title
        #context.default = c.def

        session.commit()

    #server folders --> client folders    
    for f in server_folders:
        folder = Folder()
        session.add(folder)
        folder.tid = f.id
        folder.title = f.title
        folder.archived = f.archived
        folder.private = f.private
        folder.order= f.order

        session.commit()

    #Update synch timestamps
    server_info = client.getServerInfo()

    server_sync = session.query(Sync).get('server') ###########
    #sync.timestamp = server_info.unixtime #? this doesn't work because I have decided that unixtime doesn't take into account time zones
    server_sync.timestamp = server_info.date + datetime.timedelta(seconds=5) 

    client_sync = session.query(Sync).get('client')
    client_sync.timestamp = datetime.datetime.now() + datetime.timedelta(seconds=5)

    session.commit()  

    print("New Sync times")
    print('server:', server_sync.timestamp.isoformat(b' '))
    print('client:', client_sync.timestamp.isoformat(b' '))

#@+node:slzatz.20120129221034.1645: ** OnCreateToodledoAccount - placeholder at the moment
def OnCreateToodledoAccount(self, evt=None):
    print('CreateToodledoAccount')
    #id_ = client.createAccount(email, pass_)

#@+node:slzatz.20120129221034.1646: ** get_key
def get_key():
        
    if cg.has_section('Toodledo') and cg.has_option('Toodledo', 'key') and cg.has_option('Toodledo', 'timestamp'):
        key = cg.get('Toodledo', 'key')
        date_str = cg.get('Toodledo', 'timestamp')
        try:
            key_timestamp = datetime.datetime.strptime(date_str,"%Y-%m-%dT%H:%M:%S.%f")
        except:
            key_timestamp = datetime.datetime.strptime(date_str,"%Y-%m-%dT%H:%M:%S")

        if (datetime.datetime.now() - key_timestamp) < datetime.timedelta(minutes=40, hours=3):
            print_("**used key stored in ini file**")
            # replacing print function and it was throwing exception to do this in one step
            delta = key_timestamp + datetime.timedelta(hours=4) - datetime.datetime.now()
            print_("**time left: {0} hours**".format(delta))
        else:
            key = get_new_toodledo_key()
            print_("**created new key because the key in the ini file had timed out**")

    else:
        key = get_new_toodledo_key()
        print_("**created new key because there wasn't one in ini file**")

    return key

#@+node:slzatz.20120129221034.1647: ** get_new_toodledo_key
def get_new_toodledo_key():

    if cg.has_section('Toodledo') and cg.has_option('Toodledo', 'email') and cg.has_option('Toodledo', 'pw'):
        email = cg.get('Toodledo', 'email')    

        b64pw = cg.get('Toodledo', 'pw').encode('utf-8') # type(b64pw)==bytes
        pw = base64.b64decode(b64pw).decode('utf-8') # you don't want to so str(...)
    else:
        dlg = lmdialogs.Authenticate("Toodledo Authentication", parent=None)
        if dlg.exec_(): # if cancel - dlg.exec_ is False

            email, pw = dlg.getData()

            if not cg.has_section('Toodledo'):
                cg.add_section('Toodledo')

            cg.set('Toodledo', 'email', email)
            b64pw = base64.b64encode(pw.encode('utf-8')) # type(b64pw)==bytes

            cg.set('Toodledo', 'pw', b64pw.decode('utf-8')) # need to write a decoded unicode string
        else:
            return None

    toodledo_client = toodledo.ApiClient() ### will become toodledo2
    key = toodledo_client.authenticate(email, pw, appid='mylistmanager')
    key_timestamp = datetime.datetime.now()

    cg.set('Toodledo', 'key', key)
    cg.set('Toodledo', 'timestamp', key_timestamp.isoformat())
    
    return key

#@-others
#@-leo
